HTML, XML, CSS, and XPath

library(xml2)
library(rvest)
library(purrr)
library(dplyr)
library(tibble)

url<-"https://en.wikipedia.org/wiki/List_of_mathematicians_born_in_the_19th_century"
ppl <- read_html(url) |>
    html_nodes(".mw-body-content ul li")
ppl[[1]]

from bs4 import BeautifulSoup, SoupStrainer
import urllib.request
import pandas as pd

url = "https://en.wikipedia.org/wiki/List_of_mathematicians_born_in_the_19th_century"
req = urllib.request.Request(url)
page_bytearray = urllib.request.urlopen(req)
page = page_bytearray.read()
page_bytearray.close()

soup = BeautifulSoup(page)
ppl = soup.select(".mw-body-content ul li")
ppl[0]

try_na <- function(i, fn, ...) {
    res <- try(fn(i, ...))
    if( "try-error" %in% class(res)) {
        res <- NA
    }
    if(length(res) == 0) {
        res <- NA
    }
    res
}

def try_na(x, expression):
  # If x is NA, then the result must also be NA
  # for most HTML-parsing expressions... NOT FOOLPROOF
  if pd.isna(x):
    return pd.NA
  else:
    try:
      res = eval(expression, {}, {"x": x})
    except:
      return pd.NA
    if res is None: # Tests for an empty return value
      return pd.NA
    if len(res) == 0:
      return pd.NA
  return res

math_ppl <- tibble(
    content = html_text(ppl),
    link_info = map(ppl, ~try_na(., fn = html_children)),
    name = map_chr(link_info, ~try_na(., fn = html_text)),
    name2 = map_chr(link_info, ~try_na(., fn = html_attr, "title")),
    link = map_chr(link_info, ~try_na(., fn = html_attr, "href"))
) |>
    select(-link_info)

head(math_ppl)

content = [try_na(i, "x.text") for i in ppl]
link_info = [try_na(i, "x.find('a')") for i in ppl]
name = [try_na(i, 'x.text') for i in link_info]
name2 = [try_na(i, 'x.attrs["title"]') for i in link_info]
link = [try_na(i, 'x.attrs["href"]') for i in link_info]
math_ppl = pd.DataFrame({'content': content, 'name': name, 'name2': name2, 'link': link})

math_ppl.head()